{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#| default_exp m4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#| hide\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# M4 dataset\n",
    "\n",
    "> Download and evaluate the M4 dataset.\n",
    "\n",
    "The M4 competition extended on the previous three M competitions. The dataset of the contest included 100,000 time series 95,000 of which on yearly, quarterly and monthly frequencies, and the rest in weekly, daily and hourly higher frequencies. This competition received 61 different forecasting methods, notably a neural network model outperformed the rest of the competitors, for the first time in contrast with the previous M forecasting competitions.\n",
    "\n",
    "[Spyros  Makridakis,  Evangelos  Spiliotis, and  Vassilios Assimakopoulos. The  M4  competition: 100,000  time  series and 61 forecasting methods. International Journal of Forecasting, 36(1):54–74, 2020. ISSN  0169-2070.](https://www.sciencedirect.com/science/article/pii/S0169207019301128)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "import os\n",
    "from dataclasses import dataclass\n",
    "from typing import Optional, Tuple, Union\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from utilsforecast.losses import mase, smape\n",
    "\n",
    "from datasetsforecast.utils import Info, async_download_files, download_file, extract_file"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    ">M4 meta information"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "@dataclass\n",
    "class Yearly:\n",
    "    seasonality: int = 1\n",
    "    horizon: int = 6\n",
    "    freq: str = 'Y'\n",
    "    name: str = 'Yearly'\n",
    "    n_ts: int = 23_000\n",
    "\n",
    "@dataclass\n",
    "class Quarterly:\n",
    "    seasonality: int = 4\n",
    "    horizon: int = 8\n",
    "    freq: str = 'Q'\n",
    "    name: str = 'Quarterly'\n",
    "    n_ts: int = 24_000\n",
    "\n",
    "@dataclass\n",
    "class Monthly:\n",
    "    seasonality: int = 12\n",
    "    horizon: int = 18\n",
    "    freq: str = 'M'\n",
    "    name: str = 'Monthly'\n",
    "    n_ts: int = 48_000\n",
    "\n",
    "@dataclass\n",
    "class Weekly:\n",
    "    seasonality: int = 1\n",
    "    horizon: int = 13\n",
    "    freq: str = 'W'\n",
    "    name: str = 'Weekly'\n",
    "    n_ts: int = 359\n",
    "        \n",
    "@dataclass\n",
    "class Daily:\n",
    "    seasonality: int = 1\n",
    "    horizon: int = 14\n",
    "    freq: str = 'D'\n",
    "    name: str = 'Daily'\n",
    "    n_ts: int = 4_227\n",
    "\n",
    "@dataclass\n",
    "class Hourly:\n",
    "    seasonality: int = 24\n",
    "    horizon: int = 48\n",
    "    freq: str = 'H'\n",
    "    name: str = 'Hourly'\n",
    "    n_ts: int = 414\n",
    "\n",
    "        \n",
    "@dataclass\n",
    "class Other:\n",
    "    seasonality: int = 1\n",
    "    horizon: int = 8\n",
    "    freq: str = 'D'\n",
    "    name: str = 'Other'\n",
    "    n_ts: int = 5_000\n",
    "    included_groups: Tuple = ('Weekly', 'Daily', 'Hourly')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "M4Info = Info((Yearly, Quarterly, Monthly, Weekly, Daily, Hourly, Other))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Download data class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "@dataclass\n",
    "class M4:\n",
    "    \n",
    "    source_url: str = 'https://raw.githubusercontent.com/Mcompetitions/M4-methods/master/Dataset/'\n",
    "    naive2_forecast_url: str = 'https://github.com/Nixtla/m4-forecasts/raw/master/forecasts/submission-Naive2.zip'\n",
    "\n",
    "    @staticmethod\n",
    "    def load(directory: str,\n",
    "             group: str,\n",
    "             cache: bool = True) -> Tuple[pd.DataFrame, \n",
    "                                          Optional[pd.DataFrame], \n",
    "                                          Optional[pd.DataFrame]]:\n",
    "        \"\"\"Downloads and loads M4 data.\n",
    "\n",
    "        Parameters\n",
    "        ----------\n",
    "        directory: str\n",
    "            Directory where data will be downloaded.\n",
    "        group: str\n",
    "            Group name.\n",
    "            Allowed groups: 'Yearly', 'Quarterly', 'Monthly', \n",
    "                            'Weekly', 'Daily', 'Hourly'.\n",
    "        cache: bool\n",
    "            If `True` saves and loads \n",
    "            \n",
    "        Returns\n",
    "        -------\n",
    "        df: pd.DataFrame\n",
    "            Target time series with columns ['unique_id', 'ds', 'y'].  \n",
    "        S_df: pd.DataFrame\n",
    "            Static exogenous variables with columns ['unique_id', 'ds']. \n",
    "            and static variables.       \n",
    "        \"\"\"\n",
    "        path = f'{directory}/m4/datasets'\n",
    "        file_cache = f'{path}/{group}.p'\n",
    "        \n",
    "        if os.path.exists(file_cache) and cache:\n",
    "            df, X_df, S_df = pd.read_pickle(file_cache)\n",
    "            \n",
    "            return df, X_df, S_df\n",
    "        \n",
    "        if group == 'Other':\n",
    "            #Special case.\n",
    "            included_dfs = [M4.load(directory, gr) \\\n",
    "                            for gr in M4Info['Other'].included_groups]\n",
    "            df, *_ = zip(*included_dfs)\n",
    "            df = pd.concat(df)\n",
    "        else:\n",
    "            M4.download(directory, group)\n",
    "            path = f'{directory}/m4/datasets'\n",
    "            class_group = M4Info[group]\n",
    "            S_df = pd.read_csv(f'{directory}/m4/datasets/M4-info.csv', \n",
    "                               usecols=['M4id','category'])\n",
    "            S_df['category'] = S_df['category'].astype('category').cat.codes\n",
    "            S_df.rename({'M4id': 'unique_id'}, axis=1, inplace=True)\n",
    "            S_df = S_df[S_df['unique_id'].str.startswith(class_group.name[0])]\n",
    "\n",
    "            def read_and_melt(file):\n",
    "                df = pd.read_csv(file)\n",
    "                df.columns = ['unique_id'] + list(range(1, df.shape[1]))\n",
    "                df = pd.melt(df, id_vars=['unique_id'], var_name='ds', value_name='y')\n",
    "                df = df.dropna()\n",
    "\n",
    "                return df\n",
    "\n",
    "            df_train = read_and_melt(file=f'{path}/{group}-train.csv')\n",
    "            df_test = read_and_melt(file=f'{path}/{group}-test.csv')\n",
    "\n",
    "            len_train = df_train.groupby('unique_id').agg({'ds': 'max'}).reset_index()\n",
    "            len_train.columns = ['unique_id', 'len_serie']\n",
    "            df_test = df_test.merge(len_train, on=['unique_id'])\n",
    "            df_test['ds'] = df_test['ds'] + df_test['len_serie']\n",
    "            df_test.drop('len_serie', axis=1, inplace=True)\n",
    "\n",
    "            df = pd.concat([df_train, df_test])\n",
    "            df = df.sort_values(['unique_id', 'ds']).reset_index(drop=True)\n",
    "            \n",
    "            S_df = S_df.sort_values('unique_id').reset_index(drop=True)\n",
    "        \n",
    "        X_df = None\n",
    "        if cache:\n",
    "            pd.to_pickle((df, X_df, S_df), file_cache)\n",
    "            \n",
    "        return df, None, S_df\n",
    "    \n",
    "    \n",
    "    @staticmethod\n",
    "    def _download_urls(group):\n",
    "        urls = []\n",
    "        groups = M4Info.groups if group is None else [group]\n",
    "        for group in groups:\n",
    "            for split in ('train', 'test'):\n",
    "                urls.append(f'{M4.source_url}/{split.capitalize()}/{group}-{split}.csv')\n",
    "        urls.extend([f'{M4.source_url}/M4-info.csv', M4.naive2_forecast_url])\n",
    "        return urls\n",
    "\n",
    "    @staticmethod\n",
    "    def _missing_files(path, group):\n",
    "        files = []\n",
    "        for url in M4._download_urls(group):\n",
    "            fname = url.split('/')[-1]\n",
    "            if not os.path.exists(f'{path}/{fname}'):\n",
    "                files.append(url)\n",
    "        return files\n",
    "    \n",
    "    @staticmethod\n",
    "    def _decompress(urls, path):\n",
    "        for url in urls:\n",
    "            if url.endswith('.zip'):\n",
    "                fname = url.split('/')[-1]\n",
    "                extract_file(f'{path}/{fname}', path)        \n",
    "\n",
    "    @staticmethod\n",
    "    def download(directory: str, group: Optional[str] = None) -> None:\n",
    "        \"\"\"\n",
    "        Download M4 Dataset.\n",
    "        \n",
    "        Parameters\n",
    "        ----------\n",
    "        directory: str\n",
    "            Directory path to download dataset.\n",
    "        group: str, optional (default=None)\n",
    "            Name of the group to download. If None, downloads all.\n",
    "        \"\"\"\n",
    "        path = f'{directory}/m4/datasets/'\n",
    "        missing_files = M4._missing_files(path, group)\n",
    "        if not missing_files:\n",
    "            return\n",
    "        for url in missing_files:\n",
    "            download_file(path, url)\n",
    "        M4._decompress(missing_files, path)            \n",
    "\n",
    "    @staticmethod\n",
    "    async def async_download(directory: str, group: Optional[str] = None) -> None:\n",
    "        \"\"\"\n",
    "        Download M4 Dataset.\n",
    "        \n",
    "        Parameters\n",
    "        ----------\n",
    "        directory: str\n",
    "            Directory path to download dataset.\n",
    "        \"\"\"\n",
    "        path = f'{directory}/m4/datasets/'\n",
    "        missing_files = M4._missing_files(path, group)\n",
    "        if not missing_files:\n",
    "            return\n",
    "        await async_download_files(path, missing_files)\n",
    "        M4._decompress(missing_files, path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "group = 'Hourly'\n",
    "await M4.async_download('data', group=group)\n",
    "df, *_ = M4.load(directory='data', group=group)\n",
    "n_series = len(np.unique(df.unique_id.values))\n",
    "display_str  = f'Group: {group} '\n",
    "display_str += f'n_series: {n_series}'\n",
    "print(display_str)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluation class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "class M4Evaluation:\n",
    "    \n",
    "    @staticmethod\n",
    "    def load_benchmark(directory: str, group: str,\n",
    "                       source_url: Optional[str] = None) -> np.ndarray:\n",
    "        \"\"\"Downloads and loads a bechmark forecasts.\n",
    "        \n",
    "        Parameters\n",
    "        ----------\n",
    "        directory: str\n",
    "            Directory where data will be downloaded.\n",
    "        group: str\n",
    "            Group name.\n",
    "            Allowed groups: 'Yearly', 'Quarterly', 'Monthly', \n",
    "                            'Weekly', 'Daily', 'Hourly'.\n",
    "        source_url: str, optional\n",
    "            Optional benchmark url obtained from \n",
    "            https://github.com/Nixtla/m4-forecasts/tree/master/forecasts.\n",
    "            If `None` returns Naive2. \n",
    "        \n",
    "        Returns\n",
    "        -------\n",
    "        benchmark: numpy array\n",
    "            Numpy array of shape (n_series, horizon).\n",
    "        \"\"\"\n",
    "        path = f'{directory}/m4/datasets'\n",
    "        initial = group[0]\n",
    "        if source_url is not None:\n",
    "            filename = source_url.split('/')[-1].replace('.rar', '.csv')\n",
    "            filepath = f'{path}/{filename}'\n",
    "            if not os.path.exists(filepath):\n",
    "                download_file(path, source_url, decompress=True)\n",
    "        else:\n",
    "            filepath = f'{path}/submission-Naive2.csv'\n",
    "        \n",
    "        benchmark = pd.read_csv(filepath)\n",
    "        benchmark = benchmark[benchmark['id'].str.startswith(initial)]\n",
    "        benchmark = benchmark.set_index('id').dropna(axis=1)\n",
    "        benchmark = benchmark.sort_values('id').values\n",
    "        \n",
    "        return benchmark\n",
    "    \n",
    "    @staticmethod\n",
    "    def evaluate(directory: str, group: str, \n",
    "                 y_hat: Union[np.ndarray, str]) -> pd.DataFrame:\n",
    "        \"\"\"Evaluates y_hat according to M4 methodology.\n",
    "        \n",
    "        Parameters\n",
    "        ----------\n",
    "        directory: str\n",
    "            Directory where data will be downloaded.\n",
    "        group: str\n",
    "            Group name.\n",
    "            Allowed groups: 'Yearly', 'Quarterly', 'Monthly', \n",
    "                            'Weekly', 'Daily', 'Hourly'.\n",
    "        y_hat: numpy array, str\n",
    "            Group forecasts as numpy array or\n",
    "            benchmark url from\n",
    "            https://github.com/Nixtla/m4-forecasts/tree/master/forecasts.\n",
    "            \n",
    "        Returns\n",
    "        -------\n",
    "        evaluation: pandas dataframe\n",
    "            DataFrame with columns OWA, SMAPE, MASE\n",
    "            and group as index.\n",
    "        \"\"\"\n",
    "        if isinstance(y_hat, str):\n",
    "            y_hat = M4Evaluation.load_benchmark(directory, group, y_hat)\n",
    "        \n",
    "        initial = group[0]\n",
    "        class_group = M4Info[group]\n",
    "        horizon = class_group.horizon\n",
    "        seasonality = class_group.seasonality\n",
    "        path = f'{directory}/m4/datasets'\n",
    "        y_df, *_ = M4.load(directory, group)\n",
    "\n",
    "        naive2 = M4Evaluation.load_benchmark(directory, group)\n",
    "        y_test = y_df.groupby('unique_id').tail(horizon).copy()\n",
    "        y_train = y_df.drop(y_test.index)\n",
    "        y_test['y_hat'] = y_hat.ravel()\n",
    "        y_test['naive2'] = naive2.ravel()\n",
    "\n",
    "        models = ['y_hat', 'naive2']\n",
    "        mases = mase(\n",
    "            y_test, models=models, seasonality=seasonality, train_df=y_train\n",
    "        )\n",
    "        smapes = smape(y_test, models=models)\n",
    "        avg_mases = mases[models].mean()\n",
    "        avg_smapes = smapes[models].mean()\n",
    "        mases_ratio = avg_mases['y_hat'] / avg_mases['naive2']\n",
    "        smapes_ratio = avg_smapes['y_hat'] / avg_smapes['naive2']\n",
    "        owa = 0.5 * (mases_ratio + smapes_ratio)\n",
    "        \n",
    "        evaluation = pd.DataFrame({'SMAPE': 200 * avg_smapes['y_hat'],\n",
    "                                   'MASE': avg_mases['y_hat'],\n",
    "                                   'OWA': owa},\n",
    "                                   index=[group])\n",
    "        \n",
    "        return evaluation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### URL-based evaluation\n",
    "\n",
    "The method `evaluate` from the class `M4Evaluation` can receive a url of a [benchmark uploaded to the M4 competiton](https://github.com/Mcompetitions/M4-methods/tree/master/Point%20Forecasts).  \n",
    "\n",
    "The results compared to the on-the-fly evaluation were obtained from the [official evaluation](https://github.com/Mcompetitions/M4-methods/blob/master/Evaluation%20and%20Ranks.xlsx)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastcore.test import test_close"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "esrnn_url = 'https://github.com/Nixtla/m4-forecasts/raw/master/forecasts/submission-118.zip'\n",
    "esrnn_evaluation = M4Evaluation.evaluate('data', 'Hourly', esrnn_url)\n",
    "# Test of the same evaluation as the original one\n",
    "test_close(esrnn_evaluation['SMAPE'].item(), 9.328, eps=1e-3)\n",
    "test_close(esrnn_evaluation['MASE'].item(), 0.893, eps=1e-3)\n",
    "test_close(esrnn_evaluation['OWA'].item(), 0.440, eps=1e-3)\n",
    "esrnn_evaluation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Numpy-based evaluation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Also the method `evaluate` can recevie a numpy array of forecasts.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fforma_url = 'https://github.com/Nixtla/m4-forecasts/raw/master/forecasts/submission-245.zip'\n",
    "fforma_forecasts = M4Evaluation.load_benchmark('data', 'Hourly', fforma_url)\n",
    "fforma_evaluation = M4Evaluation.evaluate('data', 'Hourly', fforma_forecasts)\n",
    "# Test of the same evaluation as the original one\n",
    "test_close(fforma_evaluation['SMAPE'].item(), 11.506, eps=1e-3)\n",
    "test_close(fforma_evaluation['MASE'].item(), 0.819, eps=1e-3)\n",
    "test_close(fforma_evaluation['OWA'].item(), 0.484, eps=1e-3)\n",
    "fforma_evaluation"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "python3",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
